

# 打开源文件，得到所有的数据,怎么读取可以参照
print("这里是什么")
def open_train_data(file_path_in,file_path_out):
    print("进入open_train_data")
    with open(file_path_in, 'r', encoding="utf8", errors="ignore") as data_f:
        with open(file_path_out, 'w', encoding="utf8", errors="ignore") as data_o:
            print("读取文件：",file_path_in)
            print("写入文件：",file_path_out)
            irotate = 0
            new_pair_num = 0
            src_last = ''
            tgt_list = []
            for line in data_f:
                data = line.strip().split('\t')
                if len(data) != 4:
                    print("miss: "+line.strip())
                    continue
                src, tgt, keyword, cluster = data[0], data[1], data[2], data[3]
                if len(src) == 0 or len(tgt) == 0:
                    print("miss: %s,%s"%(src,tgt))
                    continue
                # 整合相同上文的
                if src == src_last:
                    tgt_list.append(tgt)
                else:
                    if irotate!=0:
                        # 上个话题结束，写入文件中
                        # print(len(tgt_list)) # 显示该src具有多少tgt
                        data_write = src_last +'\t'+str(tgt_list)+'\n'
                        data_o.writelines(data_write)
                        new_pair_num = new_pair_num + 1

                    # 开始一个新的话题
                    src_last = src
                    tgt_list = [tgt]

                if irotate>=1000:
                    break
                irotate = irotate + 1
                # print(src,"\n",tgt)

            data_write = src + '\t' + str(tgt_list) + '\n'
            data_o.writelines(data_write)
            new_pair_num = new_pair_num + 1
    print(irotate,new_pair_num)



file_path_in = '/mnt/sdg/lmj/gitlucky_dcvae/dcvae/multi-gen/data/weibo_src_tgt_keyword_label.train'
file_path_out = '/mnt/sdg/lmj/gitlucky_dcvae/dcvae/multi-gen/data/data_fix/weibo_all'
open_train_data(file_path_in,file_path_out)